運用を自動化！ CloudWatch と Systems Manager オートメーションでプロセス監視＆自動復旧をやってみた

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "ActionForSSMAutomation",
            "Action": "ssm:StartAutomationExecution",
            "Effect": "Allow",
            "Resource": [
                "*"
            ]
        }
    ]
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "TrustEventBridgeService",
            "Effect": "Allow",
            "Principal": {
                "Service": [
                    "ssm.amazonaws.com",
                    "events.amazonaws.com"
                ]
            },
            "Action": "sts:AssumeRole"
        }
    ]
}

{
	"Version": "2012-10-17",
	"Statement": [
		{
			"Sid": "VisualEditor0",
			"Effect": "Allow",
			"Action": "iam:PassRole",
			"Resource": "arn:aws:iam::XXXXXXXXXXX:role/Automation-Execute-Role"
		}
	]
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "",
            "Effect": "Allow",
            "Principal": {
                "Service": "ssm.amazonaws.com"
            },
            "Action": "sts:AssumeRole"
        }
    ]
}

schemaVersion: '0.3'
description: This book that automatically recovers services triggered by CloudWatch alarm state changes.
parameters:
  pattern:
    type: String
    description: Target Service Name.
  InstanceId:
    type: StringList
    description: Target InstanceId.
assumeRole: arn:aws:iam::XXXXXXXXXXX:role/Automation-Execute-Role
mainSteps:
  - name: CheckAndStartService
    action: aws:runCommand
    isEnd: true
    inputs:
      DocumentName: AWS-RunShellScript
      Parameters:
        commands:
          - '#!/bin/bash'
          - SERVICE_NAME="{{ pattern }}"
          - '# Check the status of the service'
          - if systemctl is-active --quiet "$SERVICE_NAME"; then
          - '  echo "$SERVICE_NAME is already running."'
          - else
          - '  echo "$SERVICE_NAME is stopped. Starting it..."'
          - '  systemctl start "$SERVICE_NAME"'
          - '  # Recheck the status after starting'
          - '  if systemctl is-active --quiet "$SERVICE_NAME"; then'
          - '    echo "Successfully started service $SERVICE_NAME."'
          - '  else'
          - '    echo "Failed to start service $SERVICE_NAME."'
          - '    exit 1'
          - '  fi'
          - fi
      InstanceIds: '{{ InstanceId }}'

cd /opt/aws/amazon-cloudwatch-agent/bin/
vi config.json

{
    "agent": {
      "metrics_collection_interval": 60,
      "run_as_user": "cwagent"
    },
    "metrics": {
      "metrics_collected": {
        "procstat": [
          {
            "pattern": "nginx",
            "measurement": [
              "pid_count"
            ],
            "metrics_collection_interval": 60
          },
          {
            "pattern": "sshd",
            "measurement": [
              "pid_count"
            ],
            "metrics_collection_interval": 60
          },
          {
            "pattern": "mysqld",
            "measurement": [
              "pid_count"
            ],
            "metrics_collection_interval": 60
          }
        ]
      },
      "append_dimensions": {
        "InstanceId": "${aws:InstanceId}"
      }
    }
  }

sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/bin/config.json

{
    "source": ["aws.cloudwatch"],
    "detail-type": ["CloudWatch Alarm State Change"],
    "detail": {
      "state": {
        "value": ["ALARM"]
      }
    },
    "resources": [
      "arn:aws:cloudwatch:ap-northeast-1:XXXXXXXXXXX:alarm:WebServerProcessMonitor-nginx",
      "arn:aws:cloudwatch:ap-northeast-1:XXXXXXXXXXX:alarm:WebServerProcessMonitor-sshd",
      "arn:aws:cloudwatch:ap-northeast-1:XXXXXXXXXXX:alarm:WebServerProcessMonitor-mysqld"
    ]
}

{
    "InstanceId": "$.detail.configuration.metrics[0].metricStat.metric.dimensions.InstanceId",
    "pattern": "$.detail.configuration.metrics[0].metricStat.metric.dimensions.pattern"
}

{
    "pattern": ["<pattern>"],
    "InstanceId":["<InstanceId>"]
}

sudo systemctl stop nginx
sudo systemctl stop sshd
sudo systemctl stop mysqld

aws sqs receive-message \
    --queue-url https://sqs.ap-northeast-1.amazonaws.com/XXXXXXXXXXX/XXX \
    --max-number-of-messages 10 \
    --visibility-timeout 30 \
    --wait-time-seconds 0 \
    --attribute-names All \
    --message-attribute-names All \
    --output json

{
    "Messages": [
        {
            "MessageId": "5cd169b7-2e1b-47f0-8c5a-28de9432ff0e",
            "ReceiptHandle": "AQEBSDF0tE7hMvNm+WHeTedJ2IlmeMMywObJ9qudb8cu97Cd78gxVy2SjJ0NdOvLqTfDT7UG2sUtXu2mvskDMP7YeK+FBtQazc0cAlE9sxXcFXpuKms06TzxirjBxlKeNVf5USZSP2biyBOEb88XF2HSCxZ1uSddRuEzYyU7mYjJcRfCk9uJJhwgy6c/3k1AB+ThJ+JAHmr4wTZBj0XG1lU1adnGWnNZWo6VgDYbHMomk/ArAJAhqU5ScAcQruS+i0PXmxaofmSF1X6Cl+hKTOJK0hJOqn92BB3wxT81bvIioZ63EuSGKj+W9zdGLmCrtEZUoV66MS5DDiaAf5MLNLF6HJjwbicNzNCzObqaJHJ2y75g/SRrT3BQiMSnfT+0fKgOIrdZY5Rwc5DgAtwiwZdhMw==",
            "MD5OfBody": "9af3633b723de6ef7ac9c2e3012ad176",
            "Body": "{\n  \"pattern\": [\"nginx\"],\n  \"InstanceId\":[\"i-XXXXXXXXXXX\"]\n}",
            "Attributes": {
                "SenderId": "AIDAIVNDY5GZ7FOG4K4K2",
                "ApproximateFirstReceiveTimestamp": "1728291225293",
                "ApproximateReceiveCount": "4",
                "SentTimestamp": "1728011221747"
            },
            "MD5OfMessageAttributes": "af27f0c1b7cd363a38347b74ff975a3a",
            "MessageAttributes": {
                "ERROR_CODE": {
                    "StringValue": "INVALID_PARAMETER",
                    "DataType": "String"
                },
                "ERROR_MESSAGE": {
                    "StringValue": "The defined assume role is unable to be assumed. (Service: AWSSimpleSystemsManagement; Status Code: 400; Error Code: InvalidAutomationExecutionParametersException; Request ID: 0a022d16-7d89-46dc-9d76-1347484d4909; Proxy: null)",
                    "DataType": "String"
                },
                "RULE_ARN": {
                    "StringValue": "arn:aws:events:ap-northeast-1:XXXXXXXXXXX:rule/nginx-test",
                    "DataType": "String"
                },
                "TARGET_ARN": {
                    "StringValue": "arn:aws:ssm:ap-northeast-1:XXXXXXXXXXX:automation-definition/linux-recovery",
                    "DataType": "String"
                }
            }
        }
    ]
}